﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Data.SqlClient;
using System.Data;

namespace Common.TextMatch
{
    /// <summary>
    /// 
    /// </summary>
    public class TfidfIndexer
    {
        #region settings
        private int MinTermFrequency = 4;
        private int MaxWordLength = 30;
        #endregion

        #region members
        private string _ConnStr;
        /// <summary></summary>
        public string ConnStr
        {
            get
            {
                return this._ConnStr;
            }
            set
            {
                this._ConnStr = value;
            }
        }

        private string _IndexTableName;
        /// <summary></summary>
        public string IndexTableName
        {
            get
            {
                return this._IndexTableName;
            }
            set
            {
                this._IndexTableName = value;
            }
        }

        private string _SrcTblName;
        /// <summary></summary>
        public string SrcTblName
        {
            get
            {
                return this._SrcTblName;
            }
            set
            {
                this._SrcTblName = value;
            }
        }

        private string _SrcFieldName;
        /// <summary></summary>
        public string SrcFieldName
        {
            get
            {
                return this._SrcFieldName;
            }
            set
            {
                this._SrcFieldName = value;
            }
        }
        #endregion

        /// <summary>
        /// 
        /// </summary>
        /// <param name="connStr"></param>
        /// <param name="srcTblName"></param>
        /// <param name="srcColName"></param>
        /// <param name="indexTblName"></param>
        public TfidfIndexer(string connStr, string srcTblName, string srcColName, string indexTblName)
        {
            this._ConnStr = connStr;
            this._SrcTblName = srcTblName;
            this._SrcFieldName = srcColName;
            this._IndexTableName = indexTblName;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public void IndexTerm()
        {
            SqlConnection conn = new SqlConnection(this._ConnStr);
            Dictionary<string, int> termFrequency = new Dictionary<string, int>();
            Dictionary<string, int> inverseDocFrequency = new Dictionary<string, int>();
            StopWordsUtil stopWordUtil = new StopWordsUtil(StopWordsUtil.StopWordList.PubmedStopWordList);
            try
            {
                conn.Open();
                SqlCommand cmd = conn.CreateCommand();
                cmd.CommandType = CommandType.Text;
                cmd.CommandText = string.Format("select {0} from {1}", this._SrcFieldName, this._SrcTblName);
                SqlDataReader reader = cmd.ExecuteReader();
                int docCount = 0;
                while (reader.Read())
                {
                    if (reader[0] != null && reader[0] != DBNull.Value)
                    {
                        docCount++;
                        string phrase = reader.GetString(0);
                        string[] terms = StringUtil.Split(phrase);
                        List<string> distinctTerms = new List<string>();
                        foreach (string term in terms)
                        {
                            string word = term.Trim().ToUpper();
                            if (word.Length > 0 && word.Length <= MaxWordLength)
                            {
                                if (!stopWordUtil.IsStopWord(word))
                                {
                                    if (termFrequency.ContainsKey(word))
                                    {
                                        termFrequency[word] = termFrequency[word] + 1;
                                    }
                                    else
                                    {
                                        termFrequency.Add(word, 1);
                                    }
                                    if (!distinctTerms.Contains(word))
                                        distinctTerms.Add(word);
                                }
                            }

                            foreach (string distinctWord in distinctTerms)
                            {
                                if (inverseDocFrequency.ContainsKey(distinctWord))
                                {
                                    inverseDocFrequency[distinctWord] = inverseDocFrequency[distinctWord] + 1;
                                }
                                else
                                {
                                    inverseDocFrequency.Add(distinctWord, 1);
                                }
                            }
                        }
                    }
                }
                reader.Close();

                List<TableColumn> cols = new List<TableColumn>();
                TableColumn col1 = new TableColumn();
                col1.AllowNull = false;
                col1.ColumnName = "Term";
                col1.DbType = SqlDbType.VarChar;
                col1.Size = 30;
                cols.Add(col1);
                TableColumn col2 = new TableColumn();
                col1.AllowNull = false;
                col1.ColumnName = "Score";
                col1.DbType = SqlDbType.Float;
                col1.Size = FieldDataType.GetFieldDefaultSize(SqlDbType.Float);
                cols.Add(col2);
                TableHelper.CreateTable(this._ConnStr, this._IndexTableName, cols.ToArray(), true);

                string spName = "BulkInsert_" + this._IndexTableName + "_SP";
                string spBody = StoredProcUtil.CreateStoredProcedureStatementForInsert(DataProviderType.MSSQL, spName,
                                                                                       string.Empty,
                                                                                       this._IndexTableName,
                                                                                       cols.ToArray());
                DbUtil.NewStoredProc(this._ConnStr, spName, spBody);
                SqlParameter termParam = new SqlParameter("@" + col1.ColumnName, DBNull.Value);
                SqlParameter scoreParam = new SqlParameter("@" + col2.ColumnName, DBNull.Value);
                cmd.CommandText = spName;
                cmd.CommandType = CommandType.StoredProcedure;

                foreach (string term in termFrequency.Keys)
                {
                    if (termFrequency[term] >= MinTermFrequency)
                    {
                        double score = ((double)termFrequency[term]) /
                                       Math.Log10((double)docCount / inverseDocFrequency[term]);
                        cmd.Parameters.Clear();
                        termParam.Value = term;
                        scoreParam.Value = score;
                        cmd.Parameters.AddRange(new SqlParameter[] { termParam, scoreParam });
                        cmd.ExecuteNonQuery();
                    }
                }
            }
            catch (Exception ex)
            {
                ExceptionHandler.HandlesException(ex);
            }
            finally
            {
                conn.Close();
            }
        }
    }
}
